{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XGBoost Parameter Tuning for RentListingInquries"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. 用GridSearchCV调整正则参数reg_alpha & reg_lambda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#导入准备调用的模块\n",
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>5465</td>\n",
       "      <td>2732.5</td>\n",
       "      <td>1821.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3275</td>\n",
       "      <td>1637.5</td>\n",
       "      <td>1637.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3350</td>\n",
       "      <td>1675.0</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 228 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.5         3   3000           1200.0      750.000000       -1.5   \n",
       "1        1.0         2   5465           2732.5     1821.666667       -1.0   \n",
       "2        1.0         1   2850           1425.0     1425.000000        0.0   \n",
       "3        1.0         1   3275           1637.5     1637.500000        0.0   \n",
       "4        1.0         4   3350           1675.0      670.000000       -3.0   \n",
       "\n",
       "   room_num  Year  Month  Day       ...        walk  walls  war  washer  \\\n",
       "0       4.5  2016      6   24       ...           0      0    0       0   \n",
       "1       3.0  2016      6   12       ...           0      0    0       0   \n",
       "2       2.0  2016      4   17       ...           0      0    0       0   \n",
       "3       2.0  2016      4   18       ...           0      0    0       0   \n",
       "4       5.0  2016      4   28       ...           0      0    1       0   \n",
       "\n",
       "   water  wheelchair  wifi  windows  work  interest_level  \n",
       "0      0           0     0        0     0               1  \n",
       "1      0           0     0        0     0               2  \n",
       "2      0           0     0        0     0               0  \n",
       "3      0           0     0        0     0               2  \n",
       "4      0           0     0        0     0               2  \n",
       "\n",
       "[5 rows x 228 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#读取train的数据文件并显示头5行数据\n",
    "train = pd.read_csv(\"./data/RentListingInquries_FE_train.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#分离特征列与目标列\n",
    "y_train = train['interest_level']\n",
    "\n",
    "X_train = train.drop(['interest_level'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#从前序代码的countplot图可知，三类目标的样本数量不均匀，故采用分层采样，考虑时间代价，将划分等级设为3，可能会影响最终的模型参数\n",
    "kfold = StratifiedKFold(n_splits=3, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': range(0, 3), 'reg_lambda': range(0, 3)}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#将两个参数一起进行调优，设定reg_alpha，reg_lambda调整的网格\n",
    "reg_alpha = range (0,3)\n",
    "reg_lambda = range (0,3)\n",
    "param_t5=dict(reg_alpha = reg_alpha ,reg_lambda = reg_lambda )\n",
    "param_t5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xgb5_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=220,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5, #第二轮参数调优时得到的max_depth最佳值\n",
    "        min_child_weight=7, #第三轮参数调优时得到的min_child_weight最佳值\n",
    "        gamma=0,\n",
    "        subsample=0.8, #第四轮参数调优时得到的subsample最佳值\n",
    "        colsample_bytree=0.8, #第四轮参数调优时得到的colsample_bytree最佳值\n",
    "        colsample_bylevel = 0.8,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=3, random_state=3, shuffle=True),\n",
       "       error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=0.8,\n",
       "       colsample_bytree=0.8, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=5, min_child_weight=7, missing=None, n_estimators=220,\n",
       "       n_jobs=1, nthread=None, objective='multi:softprob', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=3, silent=True,\n",
       "       subsample=0.8),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'reg_alpha': range(0, 3), 'reg_lambda': range(0, 3)},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#实例化reg_alpha，reg_lambda调优的GridSearchCV,并将以上的调优取值集合与学习器代入\n",
    "gsearch5_1 = GridSearchCV(xgb5_1, param_grid = param_t5,scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch5_1.fit(X_train , y_train) #利用实例好的GridSearchCV来训练数据\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>mean_train_score</th>\n",
       "      <th>param_reg_alpha</th>\n",
       "      <th>param_reg_lambda</th>\n",
       "      <th>params</th>\n",
       "      <th>rank_test_score</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>split0_train_score</th>\n",
       "      <th>split1_test_score</th>\n",
       "      <th>split1_train_score</th>\n",
       "      <th>split2_test_score</th>\n",
       "      <th>split2_train_score</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>std_train_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>317.015466</td>\n",
       "      <td>1.139732</td>\n",
       "      <td>-0.587247</td>\n",
       "      <td>-0.505024</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>{'reg_alpha': 0, 'reg_lambda': 0}</td>\n",
       "      <td>6</td>\n",
       "      <td>-0.582847</td>\n",
       "      <td>-0.506467</td>\n",
       "      <td>-0.588311</td>\n",
       "      <td>-0.503683</td>\n",
       "      <td>-0.590584</td>\n",
       "      <td>-0.504922</td>\n",
       "      <td>16.152290</td>\n",
       "      <td>0.039197</td>\n",
       "      <td>0.003247</td>\n",
       "      <td>0.001139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>311.742831</td>\n",
       "      <td>1.206402</td>\n",
       "      <td>-0.587447</td>\n",
       "      <td>-0.507443</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>{'reg_alpha': 0, 'reg_lambda': 1}</td>\n",
       "      <td>9</td>\n",
       "      <td>-0.582717</td>\n",
       "      <td>-0.508675</td>\n",
       "      <td>-0.588449</td>\n",
       "      <td>-0.505834</td>\n",
       "      <td>-0.591176</td>\n",
       "      <td>-0.507820</td>\n",
       "      <td>7.800727</td>\n",
       "      <td>0.139522</td>\n",
       "      <td>0.003525</td>\n",
       "      <td>0.001190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>319.349599</td>\n",
       "      <td>1.065394</td>\n",
       "      <td>-0.587254</td>\n",
       "      <td>-0.508285</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>{'reg_alpha': 0, 'reg_lambda': 2}</td>\n",
       "      <td>7</td>\n",
       "      <td>-0.582501</td>\n",
       "      <td>-0.509594</td>\n",
       "      <td>-0.588555</td>\n",
       "      <td>-0.506878</td>\n",
       "      <td>-0.590705</td>\n",
       "      <td>-0.508382</td>\n",
       "      <td>18.491516</td>\n",
       "      <td>0.054393</td>\n",
       "      <td>0.003473</td>\n",
       "      <td>0.001111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>319.676284</td>\n",
       "      <td>1.143732</td>\n",
       "      <td>-0.587096</td>\n",
       "      <td>-0.505061</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>{'reg_alpha': 1, 'reg_lambda': 0}</td>\n",
       "      <td>5</td>\n",
       "      <td>-0.582890</td>\n",
       "      <td>-0.506477</td>\n",
       "      <td>-0.588483</td>\n",
       "      <td>-0.503416</td>\n",
       "      <td>-0.589914</td>\n",
       "      <td>-0.505292</td>\n",
       "      <td>12.660824</td>\n",
       "      <td>0.109399</td>\n",
       "      <td>0.003031</td>\n",
       "      <td>0.001260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>308.439308</td>\n",
       "      <td>1.061394</td>\n",
       "      <td>-0.586863</td>\n",
       "      <td>-0.506294</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>{'reg_alpha': 1, 'reg_lambda': 1}</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.582797</td>\n",
       "      <td>-0.507835</td>\n",
       "      <td>-0.587935</td>\n",
       "      <td>-0.504563</td>\n",
       "      <td>-0.589858</td>\n",
       "      <td>-0.506485</td>\n",
       "      <td>6.623410</td>\n",
       "      <td>0.038180</td>\n",
       "      <td>0.002980</td>\n",
       "      <td>0.001343</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>314.967682</td>\n",
       "      <td>1.092729</td>\n",
       "      <td>-0.587002</td>\n",
       "      <td>-0.508432</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>{'reg_alpha': 1, 'reg_lambda': 2}</td>\n",
       "      <td>3</td>\n",
       "      <td>-0.582107</td>\n",
       "      <td>-0.510111</td>\n",
       "      <td>-0.588754</td>\n",
       "      <td>-0.505853</td>\n",
       "      <td>-0.590145</td>\n",
       "      <td>-0.509332</td>\n",
       "      <td>15.053557</td>\n",
       "      <td>0.055731</td>\n",
       "      <td>0.003507</td>\n",
       "      <td>0.001851</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>318.263204</td>\n",
       "      <td>1.120731</td>\n",
       "      <td>-0.587006</td>\n",
       "      <td>-0.507087</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>{'reg_alpha': 2, 'reg_lambda': 0}</td>\n",
       "      <td>4</td>\n",
       "      <td>-0.582441</td>\n",
       "      <td>-0.508216</td>\n",
       "      <td>-0.588630</td>\n",
       "      <td>-0.505375</td>\n",
       "      <td>-0.589947</td>\n",
       "      <td>-0.507669</td>\n",
       "      <td>17.090196</td>\n",
       "      <td>0.050680</td>\n",
       "      <td>0.003272</td>\n",
       "      <td>0.001231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>323.905860</td>\n",
       "      <td>1.180734</td>\n",
       "      <td>-0.586880</td>\n",
       "      <td>-0.509035</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>{'reg_alpha': 2, 'reg_lambda': 1}</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.582509</td>\n",
       "      <td>-0.510074</td>\n",
       "      <td>-0.588816</td>\n",
       "      <td>-0.507415</td>\n",
       "      <td>-0.589315</td>\n",
       "      <td>-0.509617</td>\n",
       "      <td>14.275819</td>\n",
       "      <td>0.047397</td>\n",
       "      <td>0.003098</td>\n",
       "      <td>0.001161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>280.470375</td>\n",
       "      <td>0.899385</td>\n",
       "      <td>-0.587330</td>\n",
       "      <td>-0.510637</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>{'reg_alpha': 2, 'reg_lambda': 2}</td>\n",
       "      <td>8</td>\n",
       "      <td>-0.582775</td>\n",
       "      <td>-0.511290</td>\n",
       "      <td>-0.588998</td>\n",
       "      <td>-0.509158</td>\n",
       "      <td>-0.590218</td>\n",
       "      <td>-0.511463</td>\n",
       "      <td>10.591976</td>\n",
       "      <td>0.074558</td>\n",
       "      <td>0.003259</td>\n",
       "      <td>0.001048</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean_fit_time  mean_score_time  mean_test_score  mean_train_score  \\\n",
       "0     317.015466         1.139732        -0.587247         -0.505024   \n",
       "1     311.742831         1.206402        -0.587447         -0.507443   \n",
       "2     319.349599         1.065394        -0.587254         -0.508285   \n",
       "3     319.676284         1.143732        -0.587096         -0.505061   \n",
       "4     308.439308         1.061394        -0.586863         -0.506294   \n",
       "5     314.967682         1.092729        -0.587002         -0.508432   \n",
       "6     318.263204         1.120731        -0.587006         -0.507087   \n",
       "7     323.905860         1.180734        -0.586880         -0.509035   \n",
       "8     280.470375         0.899385        -0.587330         -0.510637   \n",
       "\n",
       "  param_reg_alpha param_reg_lambda                             params  \\\n",
       "0               0                0  {'reg_alpha': 0, 'reg_lambda': 0}   \n",
       "1               0                1  {'reg_alpha': 0, 'reg_lambda': 1}   \n",
       "2               0                2  {'reg_alpha': 0, 'reg_lambda': 2}   \n",
       "3               1                0  {'reg_alpha': 1, 'reg_lambda': 0}   \n",
       "4               1                1  {'reg_alpha': 1, 'reg_lambda': 1}   \n",
       "5               1                2  {'reg_alpha': 1, 'reg_lambda': 2}   \n",
       "6               2                0  {'reg_alpha': 2, 'reg_lambda': 0}   \n",
       "7               2                1  {'reg_alpha': 2, 'reg_lambda': 1}   \n",
       "8               2                2  {'reg_alpha': 2, 'reg_lambda': 2}   \n",
       "\n",
       "   rank_test_score  split0_test_score  split0_train_score  split1_test_score  \\\n",
       "0                6          -0.582847           -0.506467          -0.588311   \n",
       "1                9          -0.582717           -0.508675          -0.588449   \n",
       "2                7          -0.582501           -0.509594          -0.588555   \n",
       "3                5          -0.582890           -0.506477          -0.588483   \n",
       "4                1          -0.582797           -0.507835          -0.587935   \n",
       "5                3          -0.582107           -0.510111          -0.588754   \n",
       "6                4          -0.582441           -0.508216          -0.588630   \n",
       "7                2          -0.582509           -0.510074          -0.588816   \n",
       "8                8          -0.582775           -0.511290          -0.588998   \n",
       "\n",
       "   split1_train_score  split2_test_score  split2_train_score  std_fit_time  \\\n",
       "0           -0.503683          -0.590584           -0.504922     16.152290   \n",
       "1           -0.505834          -0.591176           -0.507820      7.800727   \n",
       "2           -0.506878          -0.590705           -0.508382     18.491516   \n",
       "3           -0.503416          -0.589914           -0.505292     12.660824   \n",
       "4           -0.504563          -0.589858           -0.506485      6.623410   \n",
       "5           -0.505853          -0.590145           -0.509332     15.053557   \n",
       "6           -0.505375          -0.589947           -0.507669     17.090196   \n",
       "7           -0.507415          -0.589315           -0.509617     14.275819   \n",
       "8           -0.509158          -0.590218           -0.511463     10.591976   \n",
       "\n",
       "   std_score_time  std_test_score  std_train_score  \n",
       "0        0.039197        0.003247         0.001139  \n",
       "1        0.139522        0.003525         0.001190  \n",
       "2        0.054393        0.003473         0.001111  \n",
       "3        0.109399        0.003031         0.001260  \n",
       "4        0.038180        0.002980         0.001343  \n",
       "5        0.055731        0.003507         0.001851  \n",
       "6        0.050680        0.003272         0.001231  \n",
       "7        0.047397        0.003098         0.001161  \n",
       "8        0.074558        0.003259         0.001048  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(gsearch5_1.cv_results_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.586863 using {'reg_alpha': 1, 'reg_lambda': 1}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAELCAYAAAAoUKpTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3X14VNW96PHvL5PJG0GlRDxBUGhR\n3gRREeStB/RGAvVSkD5oKVbwhYoiKMUiVfSoj8ee0/poEXs4cqUghysKCkXl3UK5YCQkEDSgEKop\nIB7FcIIESTKTWfePvWeyZzKTmSQzCZLf53nmmT1rr732b3Ym67dfZvYSYwxKKaVUfZJaOgCllFLn\nPk0WSimlotJkoZRSKipNFkoppaLSZKGUUioqTRZKKaWi0mShlFIqKk0WSimlotJkoZRSKqrklg4g\nXrKyskyXLl1aOgyllPpeKSws/MYYc3G0eudNsujSpQsFBQUtHYZSSn2viMg/Yqmnp6GUUkpFpclC\nKaVUVJoslFJKRaXJQimlVFSaLJRSSkWlyUIppVRUmiyUUkpFdd78zuJcZIzBGDD2NPinwWDNq61b\nW2acy9vz/IXOOuHaJGj5CG2a2vnh2nQuT53lHfVDlm9Sm873SvB7Cby3wHRwm4SNqW6bBMUUvs2w\nf48wbRJ2+bplQX9kABHEesKa8k9bz2CXB8rsOv56IcsRrg7iaMtfL7TM30pIu/XVsdvwlxGmTCT0\nde26g+qE2QZBz/5lIm6n0LLI26D2PYZ/P4HWwm67ussghK0TGjshy4VuA4wJvA78D9j/64G+w2eC\nX5vwr5NcSWS2SyWRWn2yKD1RxqyFrwD+f3QJ6hwCzwZrHmCEoE45MC1g/DPturWcryVoOSPB9YLa\nxNleUEX7tXFMO+OWoPdgrSQ0NgHxd8qBj3GY5WpjtOqJ3QnWfU91Xzti9m8kx3K1edC5/pAYHdu+\nzrqM/z8wuM3ajSrB7ZuQmIK2hbVc2L+bkTpx1o3P1MYm4ni/9hLG/+6dnZ6pfW38f2VHx4exOhkT\nvIUC8+zlAuViLV/bpr99CdoSoY9AuYlQTpjYItU1BHWMSYH4JHi9pnazO9vCjlWQMO87uA1M3fLa\nbR1clhRYJiSOwDqdcTjfZ0jbQdvX+bcKjVeCyp1lSUGfwaYrb1PJY8+PjmuboVp9svB5qrnpi4Et\nHYZS5zUfPoxYuwYGUzstBoMPwJ42gWfwYcTeiROff0k7P1vtOZchaFnHtL8OwfV9hMyvs6w/Zhzz\nfXb9kLjCrLfeWEK2RbiyoDjAsayPwBGsPT894yJAk0VCdbnkEn7x1IWB1ybktEHgZeBoI+SwI7Qe\n/sNDX+2pDAz4aqetQ0j/dO16A/PsZ7AOQwMfTGPw+cuNqf3A+BzTgbb978W/juBy/2me4GWcy/kP\ncZ3rxxEjtfMc56BMSHu12yY0LgKH04Ft5ngPte+RwPYjNL7Q9xD0Xh1/L+d6HX+32lNZwX8HwrUT\ndIrK1DYT0oa1PqnThoTs3gaddnE8nK/tAybrNIf/6qK9++s/7UKSvRcsWHXqLGutRMJO17bvXHft\na7GOVALneoLrioAkSW0sjjIrBus5SWrrBE7qiP9VEmJ3Q6F1nHVDy5x1A+sPbKIwdYLWG1wnUtvO\n9urEFvp+6onfuY466w9XJ0x7EddrP6W50ki0Vp8sklxJXHRJRkuHoZRS5zT9NpRSSqmoNFkopZSK\nSpOFUkqpqDRZKKWUikqThVJKqag0WSillIpKk4VSSqmoNFkopZSKSpOFUkqpqDRZKKWUiiqhyUJE\nckXkoIgcFpFHw8yfLCInRKTIftzjmPfvIrJfRD4RkfkSesMWpZRSzSZh94YSERfwMpADHAN2i8ha\nY8yBkKpvGGOmhyw7GBgC9LWLdgD/DGxLVLxKKaUiS+SRxQDgsDHmM2NMNbAC+GmMyxogDUgBUgE3\n8FVColRKKRVVIpPFpcBRx+tjdlmo8SLykYisEpHOAMaYPGAr8KX92GiM+SSBsSqllKpHIpNFuGsM\nIaNA8A7QxRjTF9gCLAUQkW5AT6ATVoK5UUR+XGcFIlNFpEBECk6cOBHX4JVSStVKZLI4BnR2vO4E\nHHdWMMaUGWOq7JeLgOvs6XHAh8aYCmNMBbAeuCF0BcaYV4wx/Y0x/S+++OK4vwGllFKWRCaL3cAV\nItJVRFKA24G1zgoiku14OQbwn2o6AvyziCSLiBvr4raehlJKqRaSsG9DGWO8IjId2Ai4gMXGmP0i\n8jRQYIxZC8wQkTGAFzgJTLYXXwXcCHyMdepqgzHmnUTFqpRSqn4SOub091X//v1NQUFBS4ehlFLf\nKyJSaIzpH62e/oJbKaVUVJoslFJKRaXJQimlVFSaLJRSSkWlyUIppVRUmiyUUkpFpclCKaVUVJos\nlFJKRaXJQimlVFSaLJRSSkWlyUIppVRUmiyUUkpFpclCKaVUVJoslFJKRaXJQimlVFSaLJRSSkWl\nyUIppVRUmiyUUkpFpclCKaVUVJoslFJKRaXJQimlVFSaLJRSSkWlyUIppVRUmiyUUkpFpclCKaVU\nVJoslFJKRaXJQimlVFTJLR2AUqpleTwejh07RmVlZUuHohIoLS2NTp064Xa7G7W8JgulWrljx47R\ntm1bunTpgoi0dDgqAYwxlJWVcezYMbp27dqoNvQ0lFKtXGVlJe3bt9dEcR4TEdq3b9+ko0dNFkop\nTRStQFP/xposlFJKRaXJQinVosrLy/nTn/7UqGVffPFFvvvuuzhH1HilpaVcddVVTa4TzXPPPUe3\nbt3o3r07GzdubFJbsdJkoZRqUc2ZLGpqahq1nnPJgQMHWLFiBfv372fDhg3cf//9zfK+9NtQSqmA\np97Zz4Hj38a1zV4dL+DJ/9074vxHH32Uv//97/Tr14+cnBw6dOjAm2++SVVVFePGjeOpp57izJkz\nTJgwgWPHjlFTU8O8efP46quvOH78OCNGjCArK4utW7eGbT8zM5NZs2axceNGnn/+edLT05k1axYV\nFRVkZWWxZMkSsrOz2b17N3fffTdt2rRh6NChrF+/nuLi4rBtlpaWcscdd3DmzBkAFixYwODBg4Pq\nLFmyhNWrV1NVVcXnn3/OxIkTefLJJwErad1777188MEHXHrppfzlL38hPT2dRYsW8corr1BdXU23\nbt1YtmwZGRkZQe3+5S9/4fbbbyc1NZWuXbvSrVs38vPzGTRoUMx/k8bQIwulVIv63e9+x49+9COK\niorIycmhpKSE/Px8ioqKKCwsZPv27WzYsIGOHTuyb98+iouLyc3NZcaMGXTs2JGtW7dGTBQAZ86c\n4aqrrmLXrl0MHDiQBx98kFWrVlFYWMhdd93FY489BsCUKVNYuHAheXl5uFyuemPu0KEDmzdvZs+e\nPbzxxhvMmDEjbL38/HyWL19OUVERK1eupKCgAICSkhIeeOAB9u/fz0UXXcRbb70FwK233sru3bvZ\nt28fPXv25NVXXwVg7dq1PPHEEwB88cUXdO7cObCOTp068cUXX8S4tRtPjyyUUgH1HQE0h02bNrFp\n0yauueYaACoqKigpKWHYsGHMnj2bOXPmcMsttzBs2LCY23S5XIwfPx6AgwcPUlxcTE5ODmDt4Wdn\nZ1NeXs7p06cDRwcTJ07k3Xffjdimx+Nh+vTpFBUV4XK5OHToUNh6OTk5tG/fHrASwY4dOxg7dixd\nu3alX79+AFx33XWUlpYCUFxczOOPP055eTkVFRWMHDkSgDFjxjBmzBjA+s1EqOb4NpsmC6XUOcMY\nw9y5c/nVr35VZ15hYSHr1q1j7ty53HzzzYE97WjS0tICRwrGGHr37k1eXl5Qnf/5n/9pUJwvvPAC\nl1xyCfv27cPn85GWlha2Xmgn7n+dmpoaKHO5XJw9exaAyZMns2bNGq6++mqWLFnCtm3b6rTZqVMn\njh49Gnh97NgxOnbs2KD4G0NPQymlWlTbtm05ffo0ACNHjmTx4sVUVFQA1imXr7/+muPHj5ORkcGk\nSZOYPXs2e/bsqbNsLLp3786JEycCycLj8bB//37atWtH27Zt+fDDDwFYsWJFve2cOnWK7OxskpKS\nWLZsWcQLzJs3b+bkyZOcPXuWNWvWMGTIkHrbPX36NNnZ2Xg8HpYvXx62zpgxY1ixYkXgWkhJSQkD\nBgyI9tabLKHJQkRyReSgiBwWkUfDzJ8sIidEpMh+3OOYd5mIbBKRT0TkgIh0SWSsSqmW0b59e4YM\nGcJVV13F5s2bmThxIoMGDaJPnz787Gc/4/Tp03z88ccMGDCAfv368eyzz/L4448DMHXqVEaNGsWI\nESNiWldKSgqrVq1izpw5XH311fTr148PPvgAgFdffZWpU6cyaNAgjDFceOGFEdu5//77Wbp0KTfc\ncAOHDh2iTZs2YesNHTqUO+64g379+jF+/Hj69+9fb3zPPPMMAwcOJCcnhx49egTKndcsevfuzYQJ\nE+jVqxe5ubm8/PLLUa+xxIOEO/8Vl4ZFXMAhIAc4BuwGfm6MOeCoMxnob4yZHmb5bcCzxpjNIpIJ\n+IwxEb8j179/f+O/eKSUit0nn3xCz549WzqMFldRUUFmZiZgXXT/8ssv+eMf/9jo9pYsWUJBQQEL\nFiyIV4hNFu5vLSKFxpj6sxiJvWYxADhsjPnMDmgF8FPgQL1LWXV7AcnGmM0AxpiKBMaplFK89957\nPPfcc3i9Xi6//HKWLFnS0iGdUxKZLC4FjjpeHwMGhqk3XkR+jHUU8rAx5ihwJVAuIm8DXYEtwKPG\nmO//L2qUUgkxcOBAqqqqgsqWLVtGnz59Ylr+tttu47bbbgsq27hxI3PmzAkq69q1K6tXr47a3uTJ\nk5k8eXJM6/4+SGSyCPddrtBzXu8ArxtjqkTkPmApcKMd1zDgGuAI8AYwGXg1aAUiU4GpAJdddlk8\nY1dKfc/s2rUr7m2OHDky8PXV1i6RF7iPAZ0drzsBx50VjDFlxhj/rsAi4DrHsnuNMZ8ZY7zAGuDa\n0BUYY14xxvQ3xvS/+OKL4/4GlFJKWRKZLHYDV4hIVxFJAW4H1joriEi24+UY4BPHsu1ExJ8BbiSG\nax1KKaUSI2GnoYwxXhGZDmwEXMBiY8x+EXkaKDDGrAVmiMgYwAucxDrVhDGmRkRmA++L9SuWQqwj\nD6WUUi0gob/gNsasA9aFlD3hmJ4LzI2w7GagbyLjU0opFRv9BbdSqkU19hblo0ePpry8PAERxVeX\nLl345ptvmlynPhs2bKB79+5069aN3/3ud41upz6aLJRSLSpSsog2RsO6deu46KKLmrRur9fbpOXP\nBTU1NTzwwAOsX7+eAwcO8Prrr3PgQPwv8eqNBJVStdY/Cv/9cXzb/Kc+MCry3q5zPAu3201mZibZ\n2dkUFRVx4MABxo4dy9GjR6msrGTmzJlMnToVsPbGCwoKqKioYNSoUQwdOrTO+BDhDB8+nMGDB7Nz\n507GjBnDL3/5S+677z6OHDkCWAMqDRkyhBMnTjBx4kTKysq4/vrr2bBhA4WFhWRlZYVtN1KcfqWl\npeTm5jJw4ED27t3LlVdeyWuvvRYYr+Kll17inXfewePxsHLlSnr06EF+fj4PPfQQZ8+eJT09nT//\n+c907949qN38/Hy6devGD3/4QwBuv/12/vKXv9CrV68Y/jix0yMLpVSLco5n8fvf/578/HyeffbZ\nwN7x4sWLKSwspKCggPnz51NWVlanjUjjQ0RSXl7O3/72N379618zc+ZMHn74YXbv3s1bb73FPfdY\nt6h76qmnuPHGG9mzZw/jxo0LJJNIYonz4MGDTJ06lY8++ogLLrgg6IgqKyuLPXv2MG3aNP7whz8A\n0KNHD7Zv387evXt5+umn+e1vfwvA8ePHGT16NNB841vokYVSqlY9RwDNZcCAAXTt2jXwev78+YFf\nTB89epSSkpLAGBF+kcaHiMT5S+0tW7YEnbb59ttvOX36NDt27AisNzc3l3bt2tXbZixxdu7cOXDn\n2UmTJjF//nxmz54NWONd+ON/++23AevutnfeeSclJSWICB6PB4COHTuybp313aHmGt9Ck4VS6pzi\nvIPrtm3b2LJlC3l5eWRkZDB8+HAqKyvrLBNpfIhY1uHz+cjLy6tz2qohN1mNNc5I41s434PL5Qpc\nS5k3bx4jRoxg9erVlJaWMnz48DptNtf4Fg06DSUiSSJyQdyjUEq1WvWNSXHq1CnatWtHRkYGn376\naWC8iXi6+eabg+4MW1RUBFi3F3/zzTcBawS/+gZIijXOI0eOBMbSeP311xk6dGi9sZ06dYpLL70U\nIOKNDa+//npKSkr4/PPPqa6uZsWKFYFR9eIparIQkf8rIheISBusX1EfFJFH4h6JUqpVco5n8cgj\nwV1Lbm4uXq+Xvn37Mm/ePG644Ya4r3/+/PkUFBTQt29fevXqxcKFCwF48skn2bRpE9deey3r168n\nOzubtm3bhm0j1jh79uzJ0qVL6du3LydPnmTatGn1xvab3/yGuXPnMmTIkKBvhzmvWSQnJ7NgwQJG\njhxJz549mTBhAr17x3943KjjWYhIkTGmn4j8AuveTXOAQmPMOfWDOR3PQqnG0fEswquqqsLlcpGc\nnExeXh7Tpk0LHHU0RmlpKbfccgvFxcVxjLJhEj2ehVtE3MBYYIExxiMiiRkxSSmlzhFHjhxhwoQJ\n+Hw+UlJSWLSodd9xKJZk8Z9AKbAP2C4ilwPfJjIopZRqqgceeICdO3cGlc2cOZMpU6bEtPwVV1zB\n3r17g8rKysq46aab6tR9//3363zzKVSXLl1a9KiiqaImC2PMfGC+o+gfIhLbgLdKKdVCXn755bi3\n2b59+yadivo+i+UC90z7AreIyKsisgfrluFKKaVaiVi+OnuXMeZb4GbgYmAK0PK/3FFKKdVsYkkW\n/l+NjAb+bIzZR/ghU5VSSp2nYkkWhSKyCStZbBSRtoAvsWEppZQ6l8SSLO4GHgWuN8Z8B6RgnYpS\nSqkma+x4FmDdIfa7776Lc0SNV1paylVXXdXkOvUpKytjxIgRZGZmMn369Ea301BRk4Uxxgd0Ah4X\nkT8Ag40xHyU8MqVUq9CcySLaGBnfB2lpaTzzzDOBO9M2l6hfnRWR3wHXA8vtohkiMtgeElUpdR75\nt/x/49OTn8a1zR4/6MGcAXMizneOZ5GTk0OHDh148803qaqqYty4cTz11FOcOXOGCRMmcOzYMWpq\napg3bx5fffUVx48fZ8SIEWRlZbF169aw7WdmZjJr1iw2btzI888/T3p6OrNmzaKiooKsrCyWLFlC\ndnY2u3fv5u6776ZNmzYMHTqU9evXR/xdRGlpKXfccQdnzpwBYMGCBQwePDiozpIlS1i9ejVVVVV8\n/vnnTJw4kSeffBKwkta9995bZ/yNRYsW8corr1BdXU23bt1YtmxZYLwLP398hw8fjvlvEA+xnIYa\nDeQYYxYbYxYDucBPEhuWUqq1cI5nkZOTQ0lJCfn5+RQVFVFYWMj27dvZsGEDHTt2ZN++fRQXF5Ob\nm8uMGTPo2LEjW7dujZgoAM6cOcNVV13Frl27GDhwIA8++CCrVq2isLCQu+66i8ceewyAKVOmsHDh\nQvLy8nC5XPXG3KFDBzZv3syePXt44403mDFjRth6+fn5LF++nKKiIlauXIn/lkSRxt+49dZb2b17\nN/v27aNnz568+uqrAKxdu5Ynnniiwds2nmK9RflFwEl7+sIExaKUamH1HQE0h02bNrFp0yauueYa\nACoqKigpKWHYsGHMnj2bOXPmcMsttzBs2LCY23S5XIwfPx6wBh8qLi4mJycHsPbws7OzKS8v5/Tp\n04Gjg4kTJ/Luu+9GbNPj8TB9+nSKiopwuVwcOnQobL2cnJzAL7tvvfVWduzYwdixYyOOv1FcXMzj\njz9OeXk5FRUVjBw5EoAxY8Yk5E6yDRFLsngO2CsiW7G+MvtjQE9BKaXizhjD3Llz+dWvflVnXmFh\nIevWrWPu3LncfPPNMe9pp6WlBY4UjDH07t07cJtwv/puPx7OCy+8wCWXXMK+ffvw+XykpaWFrRdp\n/IpI429MnjyZNWvWcPXVV7NkyRK2bdvWoLgSKZYL3K8DNwBv249BwPYEx6WUaiWc41mMHDmSxYsX\nU1FRAVhDhn799dccP36cjIwMJk2axOzZs9mzZ0+dZWPRvXt3Tpw4EUgWHo+H/fv3065dO9q2bRsY\nh2LFihX1tnPq1Cmys7NJSkpi2bJlES+cb968mZMnT3L27FnWrFkTGCUvktOnT5OdnY3H42H58uX1\n1m1uMZ2GMsZ8Caz1vxaRI8BliQpKKdV6OMezGDVqFBMnTmTQoEGAdXH6v/7rvzh8+DCPPPIISUlJ\nuN1u/uM//gOAqVOnMmrUKLKzs+u9buGXkpLCqlWrmDFjBqdOncLr9fLQQw/Ru3dvXn31Ve69917a\ntGnD8OHDufDCyGfc77//fsaPH8/KlSsZMWJE0Mh7TkOHDuWOO+7g8OHDTJw4kf79+9c75OszzzzD\nwIEDufzyy+nTp08gEa5du5aCggKefvppwLop4bfffkt1dTVr1qxh06ZN9OrVK+r7b4qo41mEXUjk\nqDGmc/SazUfHs1CqcXQ8C0tFRQWZmZmAddH9yy+/5I9//GOj21uyZAkFBQVBo/C1tESPZxGOjmeh\nlDqvvPfeezz33HN4vV4uv/zyiMOYtlYRjyxE5CXCJwUB7jTGnFNjceuRhVKNc74cWQwcOJCqqqqg\nsmXLltGnT59Gt7lx40bmzAn+hljXrl1ZvXp1o9tsSYk6sqiv59VeWSl1Ttm1a1fc2xw5cmTg66ut\nXcRkYYxZ2pyBKKWUOnfF8gtupZRSrZwmC6WUUlFpslBKKRVVLGNwzw/zeEZEftocASqlzm+NvUX5\n6NGjKS8vT0BE8dWlSxe++eabJtepz1133UWHDh2aNE5GNLEcWaQB/YAS+9EX+AFwt4i8mLDIlFKt\nQqRkEW3siXXr1nHRRRc1ad1er7dJy58rJk+ezIYNGxK6jlh+lNcNuNEY4wUQkf8ANgE5wMcJjE0p\n1cz++1//lapP4jueRWrPHvzTb38bcb5zPAu3201mZibZ2dkUFRVx4MABxo4dy9GjR6msrGTmzJlM\nnToVsPbGCwoKqKioYNSoUQwdOrTO+BDhDB8+nMGDB7Nz507GjBnDL3/5S+677z6OHDkCWAMqDRky\nhBMnTjBx4kTKysq4/vrr2bBhA4WFhWRlZYVtN1KcfqWlpeTm5jJw4ED27t3LlVdeyWuvvRYYr+Kl\nl17inXfewePxsHLlSnr06EF+fj4PPfQQZ8+eJT09nT//+c907969zrp//OMf13sbkXiI5cjiUsB5\n45M2QEdjTA1QFX4RpZSKjXM8i9///vfk5+fz7LPPcuDAAQAWL15MYWEhBQUFzJ8/n7KysjptRBof\nIpLy8nL+9re/8etf/5qZM2fy8MMPs3v3bt566y3uueceAJ566iluvPFG9uzZw7hx4wLJJJJY4jx4\n8CBTp07lo48+4oILLgg6osrKymLPnj1MmzYtMApejx492L59O3v37uXpp5/mt3bSPX78OKNHj643\nnniL5cji34EiEdlG7S3K/1VE2gBbEhibUqqZ1XcE0FwGDBhA165dA6/nz58f+MX00aNHKSkpCYwR\n4RdpfIhIbrvttsD0li1bAokJ4Ntvv+X06dPs2LEjsN7c3FzatWtXb5uxxNm5c+fAnWcnTZrE/Pnz\nmT17NmCNd+GP/+233wasu9veeeedlJSUICJ4PB4AOnbsyLp16+qNJ96iJgtjzKsisg4YgJUsfmuM\nOW7PfiSRwSmlWh/nHVy3bdvGli1byMvLIyMjg+HDh1NZWVlnmUjjQ8SyDp/PR15eXp3TVg25yWqs\ncUYa38L5HlwuV+Bayrx58xgxYgSrV6+mtLSU4cOHxxxTvMX61dnrgWHAUOC6xIWjlGpt6huT4tSp\nU7Rr146MjAw+/fTTwHgT8XTzzTcH3Rm2qKgIsG4v/uabbwLWCH71DZAUa5xHjhwJjKXx+uuvM3To\n0HpjO3XqFJdeeilAi9/YMJavzv4OmAkcsB8zROS5WBoXkVwROSgih0Xk0TDzJ4vICREpsh/3hMy/\nQES+EJFz5x6/Sqm4co5n8cgjwScrcnNz8Xq99O3bl3nz5nHDDTfEff3z58+noKCAvn370qtXLxYu\nXAjAk08+yaZNm7j22mtZv3492dnZtG3bNmwbscbZs2dPli5dSt++fTl58iTTpk2rN7bf/OY3zJ07\nlyFDhgR9Oyz0msXPf/5zBg0axMGDB+nUqVNg7O54ijqehYh8BPQzxvjs1y5grzGmb5TlXMAhrG9N\nHQN2Az83xhxw1JkM9DfGTI/Qxh+Bi4GTker46V1nlWqc8+Wus/FWVVWFy+UiOTmZvLw8pk2bFjjq\naIzS0lJuueUWiouL4xhlwzTHeBYXASft6cjDRwUbABw2xnxmB7QC+CnW0UlUInIdcAmwAYj6RpRS\nKp6OHDnChAkT8Pl8pKSksGjRopYOqUXFkiyeA/aKyFZqvw01N4blLgWOOl4fAwaGqTdeRH6MdRTy\nsDHmqIgkAc8DdwA3xbAupZQK8sADD7Bz586gspkzZzJlypSYlr/iiivYu3dvUFlZWRk33VS3S3r/\n/ffrfPMpVJcuXVr0qKKpYvk21Ov212avx0oWc4jtwriEKQs95/UO8LoxpkpE7gOWAjcC9wPr7MQR\neQUiU4GpAJddpkOCK6Vqvfzyy3Fvs3379k06FfV9FtNpKGPMl8Ba/2sROQJE652PAc5xujsBx50V\njDHOX60sAv7Nnh4EDBOR+4FMIEVEKowxj4Ys/wrwCljXLGJ5L0oppRqusWNwR97dr7UbuEJEugJf\nALcDE4MaEcm2ExHAGOATAGPMLxx1JmNdBK/zbSqllFLNo7HJIupevDHGKyLTgY2AC1hsjNkvIk8D\nBcaYtVhfwx0DeLEuoE9uZDxKKaUSKGKyEJGXCJ8UBOvbUVEZY9YB60LKnnBMzyXKxXJjzBJgSSzr\nU0oplRj1XaguAArDPAqABxMfmlKqNWjseBZg3SH2u+++i3NEjVdaWhp1TIlY6tRn8+bNXHfddfTp\n04frrruOv/71r41uqyEiHlmB99H0AAAcyklEQVQYY5aGlonIPxlj/juxISmlWhN/srj//vsbvOyL\nL77IpEmTArf5jqampgaXy9Xg9ZxLsrKyeOedd+jYsSPFxcWMHDmSL774IuHrbeg1i3XAtYkIRCnV\n8v7fm4f45mhFXNvM6pzJsAlXRpzvHM8iJyeHDh068Oabb1JVVcW4ceN46qmnOHPmDBMmTODYsWPU\n1NQwb948vvrqK44fP86IESPIyspi69atYdvPzMxk1qxZbNy4keeff5709HRmzZpFRUUFWVlZLFmy\nhOzsbHbv3s3dd99NmzZtGDp0KOvXr4/4u4jS0lLuuOMOzpw5A8CCBQsYPHhwUJ0lS5awevVqqqqq\n+Pzzz5k4cSJPPvkkYCWte++9t874G4sWLeKVV16hurqabt26sWzZsjqJ8JprrglM9+7dm8rKSqqq\nqoJuppgIDR2DO5ZvQSmlVMyc41nk5ORQUlJCfn4+RUVFFBYWsn37djZs2EDHjh3Zt28fxcXF5Obm\nMmPGDDp27MjWrVsjJgqAM2fOcNVVV7Fr1y4GDhzIgw8+yKpVqygsLOSuu+7iscceA2DKlCksXLiQ\nvLy8qEcfHTp0YPPmzezZs4c33niDGTNmhK2Xn5/P8uXLKSoqYuXKlfhvSRRp/I1bb72V3bt3s2/f\nPnr27Bm4x9PatWt54okn6rT/1ltvcc011yQ8UUDDjyxa9+/dlTrP1XcE0Bw2bdrEpk2bAnvPFRUV\nlJSUMGzYMGbPns2cOXO45ZZbGDZsWMxtulwuxo8fD1iDDxUXF5OTkwNYe/jZ2dmUl5dz+vTpwNHB\nxIkTeffddyO26fF4mD59OkVFRbhcLg4dOhS2Xk5OTuCX3bfeeis7duxg7NixEcffKC4u5vHHH6e8\nvJyKigpGjhwJwJgxYxgzZkxQ2/v372fOnDls2rQp5m3RFA1KFsaYxl2FUkqpGBhjmDt3Lr/61a/q\nzCssLGTdunXMnTuXm2++OeyedjhpaWmBIwVjDL179w7cJtyvvtuPh/PCCy9wySWXsG/fPnw+H2lp\naWHrRRq/ItL4G5MnT2bNmjVcffXVLFmyhG3btoVt99ixY4wbN47XXnuNH/3oRw2KvbEaehpKKaXi\nyjmexciRI1m8eDEVFdZ1ky+++IKvv/6a48ePk5GRwaRJk5g9ezZ79uyps2wsunfvzokTJwLJwuPx\nsH//ftq1a0fbtm0D41CsWLGi3nZOnTpFdnY2SUlJLFu2LOj24U6bN2/m5MmTnD17ljVr1gRGyYvk\n9OnTZGdn4/F4WL58edg65eXl/OQnP+G5556L2l48abJQSrUo53gWmzdvZuLEiQwaNIg+ffrws5/9\njNOnT/Pxxx8zYMAA+vXrx7PPPsvjjz8OwNSpUxk1ahQjRoyIaV0pKSmsWrWKOXPmcPXVV9OvXz8+\n+OADAF599VWmTp3KoEGDMMZw4YWRb7B9//33s3TpUm644QYOHToUNPKe09ChQ7njjjvo168f48eP\np3//+m+g/cwzzzBw4EBycnLo0aNHoNx5zWLBggUcPnyYZ555hn79+tGvXz++/vrrmN5/U0Qdz+L7\nQsezUKpxdDwLS0VFBZmZmYB10f3LL7/kj3/8Y6PbW7JkCQUFBUGj8LW05hjPQimlzmvvvfcezz33\nHF6vl8svv7zFhzE91+iRhVKt3PlyZDFw4ECqqqqCypYtW0afPn0a3ebGjRuZM2dOUFnXrl1ZvXp1\no9tsSXpkoZRq9Xbt2hX3NkeOHBn4+mprpxe4lVJKRaXJQimlVFSaLJRSSkWlyUIppVRUmiyUUi2q\nseNZjB49mvLy8gREFF9dunThm2++aXKdSI4ePcqIESPo2bMnvXv3btJvQ+qjyUIp1aIiJYtIt9Dw\nW7duHRddFNOgnRF5vd4mLX8uSE5O5vnnn+eTTz7hww8/5OWXX+bAgQPxX0/cW1RKfW9tXfIKX//j\ns7i22eHyHzJi8tSI853jWbjdbjIzM8nOzqaoqIgDBw4wduxYjh49SmVlJTNnzmTqVKutLl26UFBQ\nQEVFBaNGjWLo0KF1xocIZ/jw4QwePJidO3cyZswYfvnLX3Lfffdx5MgRwBpQaciQIZw4cYKJEydS\nVlbG9ddfz4YNGygsLCQrKytsu5Hi9CstLSU3N5eBAweyd+9errzySl577bXAeBUvvfQS77zzDh6P\nh5UrV9KjRw/y8/N56KGHOHv2LOnp6fz5z3+me/fuQe1mZ2eTnZ0NWPfK6tmzJ1988QW9evWK4a8T\nOz2yUEq1KOd4Fr///e/Jz8/n2WefDewdL168mMLCQgoKCpg/fz5lZWV12og0PkQk5eXl/O1vf+PX\nv/41M2fO5OGHH2b37t289dZb3HPPPQA89dRT3HjjjezZs4dx48YFkkkkscR58OBBpk6dykcffcQF\nF1wQdESVlZXFnj17mDZtGn/4wx8A6NGjB9u3b2fv3r08/fTT/Pa3vwXg+PHjjB49uk77paWl7N27\nl4EDB9Yba2PokYVSKqC+I4DmMmDAALp27Rp4PX/+/MAvpo8ePUpJSUlgjAi/SONDRHLbbbcFprds\n2RJ02ubbb7/l9OnT7NixI7De3Nxc2rVrV2+bscTZuXPnwJ1iJ02axPz585k9ezZgjXfhj//tt98G\nrLvb3nnnnZSUlCAieDweADp27Mi6deuC2q6oqGD8+PG8+OKLXHDBBfXG2hiaLJRS5xTnHVy3bdvG\nli1byMvLIyMjg+HDh1NZWVlnmUjjQ8SyDp/PR15eXp3TVg25FVKscUYa38L5HlwuV+Bayrx58xgx\nYgSrV6+mtLSU4cOHh12/x+Nh/Pjx/OIXvwgknXjT01BKqRZV35gUp06dol27dmRkZPDpp58GxpuI\np5tvvjnozrBFRUWAdXvxN998E7BG8KtvgKRY4zxy5EhgLI3XX3+doUOH1hvbqVOnuPTSSwEi3tjQ\nGMPdd99Nz549mTVrVr3tNYUmC6VUi3KOZ/HII48EzcvNzcXr9dK3b1/mzZvHDTfcEPf1z58/n4KC\nAvr27UuvXr1YuHAhAE8++SSbNm3i2muvZf369WRnZ9O2bduwbcQaZ8+ePVm6dCl9+/bl5MmTTJs2\nrd7YfvOb3zB37lyGDBkS9O0w5zWLnTt3smzZMv76178GxrcIPUUVD3rXWaVaufPlrrPxVlVVhcvl\nIjk5mby8PKZNmxY46miM0tJSbrnlFoqLi+MYZcPoXWeVUirOjhw5woQJE/D5fKSkpLBo0aKWDqlF\nabJQSp2XHnjgAXbu3BlUNnPmTKZMmRLT8ldccQV79+4NKisrK+Omm26qU/f999+v882nUF26dGnR\no4qm0mShlMIYU+ebOt93L7/8ctzbbN++fZNORbWkpl5y0AvcSrVyaWlplJWVNbkzUecuYwxlZWWk\npaU1ug09slCqlevUqRPHjh3jxIkTLR2KSqC0tDQ6derU6OU1WSjVyrnd7qBfTCsVjp6GUkopFZUm\nC6WUUlFpslBKKRWVJgullFJRabJQSikVlSYLpZRSUWmyUEopFZUmC6WUUlElNFmISK6IHBSRwyLy\naJj5k0XkhIgU2Y977PJ+IpInIvtF5CMRua1u60oppZpLwn7BLSIu4GUgBzgG7BaRtcaYAyFV3zDG\nTA8p+w74pTGmREQ6AoUistEYU56oeJVSSkWWyCOLAcBhY8xnxphqYAXw01gWNMYcMsaU2NPHga+B\nixMWqVJKqXolMllcChx1vD5ml4Uab59qWiUinUNnisgAIAX4e2LCVEopFU0ik0W4m+OH3gP5HaCL\nMaYvsAVYGtSASDawDJhijPHVWYHIVBEpEJECvWOmUkolTiKTxTHAeaTQCTjurGCMKTPGVNkvFwHX\n+eeJyAXAe8DjxpgPw63AGPOKMaa/Mab/xRfrWSqllEqURCaL3cAVItJVRFKA24G1zgr2kYPfGOAT\nuzwFWA28ZoxZmcAYlVJKxSBh34YyxnhFZDqwEXABi40x+0XkaaDAGLMWmCEiYwAvcBKYbC8+Afgx\n0F5E/GWTjTHfz/EMlVLqe07Ol6EU+/fvbwoKClo6DKWU+l4RkUJjTP9o9fQX3EoppaLSZKGUUioq\nTRZKKaWi0mShlFIqKk0WSimlotJkoZRSKipNFkoppaLSZKGUUioqTRZKKaWi0mShlFIqKk0WSiml\notJkoZRSKipNFkoppaLSZKGUUioqTRZKKaWi0mShlFIqqoSNlKeUUio+fL4aajwearze2mevB5/X\ni9fjIdntpn2nyxIagyYLpVSrZ3w+amr8HbInqCP2hXTQ1rNV5vN68HpD6ng81NR463TuVl3ruU57\nYV77HOsxxldv/NndujPx2ecTuo00WSilEs4Yg6+mxuqAgzrX6B1nrHWDOnevo56j0/bVhEsAXnw1\n3ri/Z5fbjSs5GVey/ex2k5TsJjk5maRkd2C+Oy3dmu/y10km2X62lnXjcjumk5OD67jdpLe9IO7x\nh9JkodR5wuerCbtXWtu5hu9sI9etna6zl10TskxgLzvCnrPXC8bE9f0muVwhHbHdgdqdrr88JT0j\n0GnX7YiTw3bqLrszt+rFXjfZXkeSy4WIxPX9tjRNFko1gs8+ZeH1VOP1VFNTbU3XeDx4q6up8Xrq\nlIeewqjdI/bg9dR/eqLuHnndTt346j9V0VAiSYHOMSliR2l1vqnp6XX3nJODO21XSCde27m6HW2H\n7omH28t243K5kCT9fk5z0mShvneMMYGOOqhzrq6uLa+uxuuoE+jYnXU81XirPXWXtcvrLOvx2O1W\nx61jdp6eqG8P1t2mDemOPdeg+Y7pevec3W5crvo64pDOPMkVl/eozg+aLFSD+M891/g7z3Cds78D\nrlNuLxNjxx6x3ONp8vtIcrlwuVNIdrtxpVjPye4Uq7N0p5CckkJaZiauQHmKPe0mOSUlqDw5xdrj\ntcodbaU42nR26v69dVfyeXeq4rxiDBgf+GrA5wVjP/t8jmn/vOB6psYD1R58VZWYyu/wVVZiKisx\nVZX4qqoxlZX4qqow9sNXVY2prsZUVeOrrsZUefBVezD2w5r2YjxefEHPNRhPDamXdeCyVRsTujk0\nWXzPGJ8Pr9fjOL0RunccpnP2eGrLHXXqX9Z+9nrqJIJo38yISoRkR0ftSnbX6bTTMttaZckhnbNd\nJ9DRh+3wQzv24Dqteq/ZGKtTi9LZxdopNqxejV3uDZnni2O9uus1XqtD9Xm8dodbg8/rw3h8VpnX\nh8/jw3gNxmvseQZTY/DVCKZGAs+mBse0hJ/vs6YxTdgRSDIkuQzif042SBLWtMuQlAxuN0gaJCUL\nKRe3id9nJAJNFg1g7VV7rQ7W2eE6Tk/4y6y94JDOuc6edbhlPXU6c39S8J+rbqokV7K1N+xOsTrt\nFGfn68admkZ6Zts6nbO/ww7ueK29cKtTd4fsWde26dwL/97uURsD3krwnHU8vrPLvqt97XG+Pgve\ns1DjaYbOOLSePyk46jU10Tdx81kh2J2rV4KmfcaF8VkPn8+FqUnCmCR8NUmYmqRAJ2x8gs/r76Tt\nNv25w2us6UDHb8BX34X1KF2gCJLiIsmdjLhdJKW4EXcykpFMUoobV4obSXVb5SkpJKVaz5KaSlJq\nSu10WppdloqkpiHpaSSlpdvz0pH0DGs6PQNJS0eSUyApGZKSrGdxQZLLnk6CFvj/afXJorKigo0L\nX6zbaQcuUgbvWTf5Gx0iJKek1Ha67tpO23/aIiXtguDOOSVkTzoldFl/5x2mY08JaT/Zff5dGPT5\nrA45tBP3d9ShZVHrhOns/dM04u+f5AZXpH9+lz2d7Jh2dgz2tMsNyWnR68XQnpEkMILPY6wO12sw\nXl/tXrUXfJ4au8yHr7oG4/FhvDWBUx/WXnqNfTrEf4rEg6/KY5X5T6tUe6zpqipMddNOH0pqCpKW\nRlKK/ZyWiqSlWtOpqbhSrddJKam181PssrS02mm7w05KS0VSU8N05rVtkvw93bFJgFafLBA49dV/\nBzrblPQM0i8I3YOuPa0RtnMOnEqp7fAjLXs+fqUuohqvo6MN7agbsFcetHxIfW+l9WiM5DRwp4M7\nw57OsF+nQ/pFtfPc6SHzM8AdUj85Pbi+v05yOrgi/5uZmpqg89e+ykqrk62sxFdZham2y4Kmq61z\n35X+8932/Koqu50zwW3WmV9lJdjGcrutTtXRcQc66cxMklJTcKem1e2Y/Z14aoq1dx2247bnOzvv\ntDRrD721/N+co8TE+bvPLaV///6moKCgpcM49xljnRKp01HXt8cdoaP2nI3Q2dvTvsbsSUptB1xv\nR10737jSMZICSamYpFSMpGCS3CBuDCkYcWNMMkZcQLI1bQRT48N4PRivF7xejNe6cBhU5rHL6ysL\nvPZiakJe++vZe9e+qkpr2u7MacppxaSkwB5wUMft7GRTU0gK6ZglNcXek44+v07HnZqKuFrp9Z7z\nlIgUGmP6R6unRxbnCmPAWxVlrzq0M2/cXrnx1YD9RQ9jxDqN7ZOg18Yndh3rHLExSZikNKtDFrfd\nKbutTho3hrYY+QGGZMCFIRmDy1rOJIFJsto2SbXt++wYfNbFRFPjgxpfnY62tvMtx3i/sTtoT6BO\nk/aSG0IESbauLEpyMuJy2a+TkWS7zH44y5LS0pD2qbUdc2qqvScdpuO2T5eEm1+n43a7m+d9K4Um\ni+js8+Gm6jvM2W/hbAWm8jSm8jtMZQWm8gxUfWfNr7SeqTqLqT6Lqa7EVJ3FVFunEKiuxHiqrdee\narsj9GA8Hmvvs04nHaUz92F1vnanXNsh+zt4u06NYHzpGF8apuZCqGlq5+oDztqPKOwOtbYTDe5U\nxZ0MIR1tUka6da7YWe5ODilzBb922+3XV+Z2xBGpLCQ+nHV0j1q1Yq0+WXiPf84/Joy19mj9Dx+Y\nGlO752to2tfg6pVsP9Ktl4LVKblcSLL/EdJxpboRdwq4UwKdW1KYPdrwnWyMZVE73+TInaozMeh5\nZqXOC60+WUhqGqk/cCHJqcF7l+4Uq5NM8XfK9gW6lFRIsb9ZkZJunSpIyYC0dCQ1w360QVLcuveq\nlDpvtPpk4WqfTae1e1o6DKWUOqedZ1+4V0oplQiaLJRSSkWlyUIppVRUmiyUUkpFpclCKaVUVJos\nlFJKRaXJQimlVFSaLJRSSkV13tx1VkROAP9oQhNZwDdxCieeNK6G0bgaRuNqmPMxrsuNMRdHq3Te\nJIumEpGCWG7T29w0robRuBpG42qY1hyXnoZSSikVlSYLpZRSUWmyqPVKSwcQgcbVMBpXw2hcDdNq\n49JrFkoppaLSIwullFJRnffJQkRyReSgiBwWkUfDzE8VkTfs+btEpItj3ly7/KCIjGzmuGaJyAER\n+UhE3heRyx3zakSkyH6sbea4JovICcf673HMu1NESuzHnc0c1wuOmA6JSLljXiK312IR+VpEiiPM\nFxGZb8f9kYhc65iXyO0VLa5f2PF8JCIfiMjVjnmlIvKxvb0Kmjmu4SJyyvH3esIxr97PQILjesQR\nU7H9mfqBPS+R26uziGwVkU9EZL+IzAxTp3k+Y8aY8/YBuIC/Az8EUoB9QK+QOvcDC+3p24E37Ole\ndv1UoKvdjqsZ4xoBZNjT0/xx2a8rWnB7TQYWhFn2B8Bn9nM7e7pdc8UVUv9BYHGit5fd9o+Ba4Hi\nCPNHA+sBAW4AdiV6e8UY12D/+oBR/rjs16VAVgttr+HAu039DMQ7rpC6/xv4azNtr2zgWnu6LXAo\nzP9ks3zGzvcjiwHAYWPMZ8aYamAF8NOQOj8FltrTq4CbRETs8hXGmCpjzOfAYbu9ZonLGLPVGPOd\n/fJDoFOc1t2kuOoxEthsjDlpjPkfYDOQ20Jx/Rx4PU7rrpcxZjtwsp4qPwVeM5YPgYtEJJvEbq+o\ncRljPrDXC833+Yple0XSlM9mvONqzs/Xl8aYPfb0aeAT4NKQas3yGTvfk8WlwFHH62PU3dCBOsYY\nL3AKaB/jsomMy+lurD0HvzQRKRCRD0VkbJxiakhc4+3D3VUi0rmByyYyLuzTdV2BvzqKE7W9YhEp\n9kRur4YK/XwZYJOIFIrI1BaIZ5CI7BOR9SLS2y47J7aXiGRgdbhvOYqbZXuJdYr8GmBXyKxm+Yyd\n72NwS5iy0K9/RaoTy7KNFXPbIjIJ6A/8s6P4MmPMcRH5IfBXEfnYGPP3ZorrHeB1Y0yViNyHdVR2\nY4zLJjIuv9uBVcaYGkdZorZXLFri8xUzERmBlSyGOoqH2NurA7BZRD6197ybwx6s209UiMhoYA1w\nBefI9sI6BbXTGOM8Ckn49hKRTKwE9ZAx5tvQ2WEWiftn7Hw/sjgGdHa87gQcj1RHRJKBC7EOR2NZ\nNpFxISL/C3gMGGOMqfKXG2OO28+fAduw9jaaJS5jTJkjlkXAdbEum8i4HG4n5BRBArdXLCLFnsjt\nFRMR6Qv8H+Cnxpgyf7lje30NrCZ+p1+jMsZ8a4ypsKfXAW4RyeIc2F62+j5fCdleIuLGShTLjTFv\nh6nSPJ+xRFyUOVceWEdOn2GdlvBfFOsdUucBgi9wv2lP9yb4AvdnxO8CdyxxXYN1Qe+KkPJ2QKo9\nnQWUEKcLfTHGle2YHgd8aGovpn1ux9fOnv5Bc8Vl1+uOdbFRmmN7OdbRhcgXbH9C8MXH/ERvrxjj\nugzrOtzgkPI2QFvH9AdAbjPG9U/+vx9Wp3vE3nYxfQYSFZc9378j2aa5tpf93l8DXqynTrN8xuK2\noc/VB9Y3BQ5hdbyP2WVPY+2tA6QBK+1/nHzgh45lH7OXOwiMaua4tgBfAUX2Y61dPhj42P5n+Ri4\nu5njeg7Yb69/K9DDsexd9nY8DExpzrjs1/8C/C5kuURvr9eBLwEP1p7c3cB9wH32fAFetuP+GOjf\nTNsrWlz/B/gfx+erwC7/ob2t9tl/58eaOa7pjs/XhziSWbjPQHPFZdeZjPWlF+dyid5eQ7FOHX3k\n+FuNbonPmP6CWymlVFTn+zULpZRScaDJQimlVFSaLJRSSkWlyUIppVRUmiyUUkpFpclCKaVUVJos\nlGpm9m24321qHaWakyYLpULY4wPo/4ZSDvoPoRTWHT3tAWb+hHUzuztEJE9E9ojISvtGbojIaBH5\nVER22APORNz7F5EB9sBCe+3n7mHq/IuILBORv9oD1NzrmJ1p39n3UxFZbt86HxF5QkR224PwvOIv\nVyqRNFkoVas71n14crBu9/C/jDHXAgXALBFJA/4T69YvQ4GLo7T3KfBjY8w1wBPAv0ao1xfr/j6D\ngCdEpKNdfg3wENZAXD8EhtjlC4wx1xtjrgLSgVsa/E6VaqDz/RblSjXEP4wxH4rILVgd9E57pz0F\nyAN6AJ8ZazAssO4nVN/4BRcCS0XkCqz7+7gj1PuLMeYscFZEtmLdQK8c64ZwxwBEpAjrRnc7gBEi\n8hsgA+tmcfuxbh2vVMJoslCq1hn7WbBGGPu5c6aINPTW5s8AW40x4+yBa7ZFqBd6gzb/6ypHWQ2Q\nbB/d/AnrZnFHReRfsG6GqVRC6Wkoper6EBgiIt3AGh1NRK7EOq30Q7vjB7gtSjsXAl/Y05PrqfdT\nEUkTkfZYY1DvrqeuPzF8Y19H+VmUGJSKC00WSoUwxpzA6txfF5GPsJJHD/tU0f3ABhHZgXUL+VP1\nNPXvwHMishNw1VMvH3jPXs8zxh5MJ0Js5ViDTn2MNYpcfYlFqbjRW5Qr1QAikmmsIT/9YwiUGGNe\naEJ7/wJUGGP+EK8YlUoEPbJQqmHutS8278c6zfSfLRyPUs1CjyyUaiIRmQLMDCneaYx5oCXiUSoR\nNFkopZSKSk9DKaWUikqThVJKqag0WSillIpKk4VSSqmoNFkopZSK6v8DwCwStb/9o78AAAAASUVO\nRK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xc4e4828>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 将GridSerachCV得到的结果反映在图示中\n",
    "#打印最佳参数与最佳性能得分\n",
    "print(\"Best: %f using %s\" % (gsearch5_1.best_score_, gsearch5_1.best_params_))\n",
    "test_means = gsearch5_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch5_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch5_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch5_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch5_1.cv_results_).to_csv('Preds_for_regalpha_and_reglambda.csv')\n",
    "\n",
    "test_scores = np.array(test_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "train_scores = np.array(train_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "\n",
    "    \n",
    "for i, value in enumerate(reg_alpha):\n",
    "    pyplot.plot(reg_lambda, -test_scores[i], label= 'test_reg_alpha:'   + str(value))\n",
    "    pyplot.plot(reg_lambda, -train_scores[i], label= 'train_reg_alpha:'   + str(value))\n",
    "\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'reg_alpha' )                                                                                                      \n",
    "pyplot.ylabel( '-Log Loss' )\n",
    "pyplot.savefig( 'regalpha_and_reglambda1_versus_logloss.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "从.bestparams可知最佳的reg_alpha与reg_lambda值均为1"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
